{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "<h1> Ship Type Prediction </h1>\n",
    "\n",
    "<p align='justify'>\n",
    "\n",
    "This Jupyter notebook contains a classification example which is\n",
    "done with the help of Scikit-Learn library. In this notebook,\n",
    "the following steps are performed:\n",
    "</p>\n",
    "\n",
    "<ol align='justify'>\n",
    "    <li> The preprocessing i.e. feature generation, filtering and\n",
    "         interpolation of the data is carried out using the\n",
    "         PTRAIL Library.\n",
    "    </li>\n",
    "    <li> Further, several models like RandomForestClassifier, KMeans\n",
    "         Classifier etc. are trained using the Scikit-Learn library\n",
    "         based on the cleaned dataset.\n",
    "    </li>\n",
    "    <li>\n",
    "        Finally, on the interpolated dataset, the type of ships are\n",
    "        predicted and their accuracy is checked.\n",
    "    </li>"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [
    {
     "data": {
      "text/plain": "                                        lat        lon       MMSI NavStatus  \\\ntraj_id      DateTime                                                         \nAB RAMANTENN 2017-05-07 00:13:05  11.905735  57.681092  265902200    Moored   \n             2017-05-07 00:25:04  11.905740  57.681070  265902200    Moored   \n             2017-05-07 00:31:05  11.905792  57.681060  265902200    Moored   \n             2017-05-07 01:01:05  11.905650  57.681127  265902200    Moored   \n             2017-05-07 01:07:05  11.905700  57.681107  265902200    Moored   \n\n                                  SOG    COG   ShipType  \ntraj_id      DateTime                                    \nAB RAMANTENN 2017-05-07 00:13:05  0.1  170.7  Undefined  \n             2017-05-07 00:25:04  0.1  170.7  Undefined  \n             2017-05-07 00:31:05  0.1  177.4  Undefined  \n             2017-05-07 01:01:05  0.0  175.6  Undefined  \n             2017-05-07 01:07:05  0.1  180.8  Undefined  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>lat</th>\n      <th>lon</th>\n      <th>MMSI</th>\n      <th>NavStatus</th>\n      <th>SOG</th>\n      <th>COG</th>\n      <th>ShipType</th>\n    </tr>\n    <tr>\n      <th>traj_id</th>\n      <th>DateTime</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">AB RAMANTENN</th>\n      <th>2017-05-07 00:13:05</th>\n      <td>11.905735</td>\n      <td>57.681092</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>170.7</td>\n      <td>Undefined</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:25:04</th>\n      <td>11.905740</td>\n      <td>57.681070</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>170.7</td>\n      <td>Undefined</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:31:05</th>\n      <td>11.905792</td>\n      <td>57.681060</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>177.4</td>\n      <td>Undefined</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 01:01:05</th>\n      <td>11.905650</td>\n      <td>57.681127</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.0</td>\n      <td>175.6</td>\n      <td>Undefined</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 01:07:05</th>\n      <td>11.905700</td>\n      <td>57.681107</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>180.8</td>\n      <td>Undefined</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the dataset.\n",
    "\n",
    "import pandas as pd\n",
    "from ptrail.core.TrajectoryDF import PTRAILDataFrame\n",
    "\n",
    "pdf = pd.read_csv('./data/ships.csv')\n",
    "np_ships = PTRAILDataFrame(data_set=pdf,\n",
    "                           latitude='lat',\n",
    "                           longitude='lon',\n",
    "                           datetime='Timestamp',\n",
    "                           traj_id='Name')\n",
    "np_ships.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 195 ms, sys: 8.04 ms, total: 203 ms\n",
      "Wall time: 201 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": "                                        lat        lon       MMSI NavStatus  \\\ntraj_id      DateTime                                                         \nAB RAMANTENN 2017-05-07 00:13:05  11.905735  57.681092  265902200    Moored   \n             2017-05-07 00:25:04  11.905740  57.681070  265902200    Moored   \n             2017-05-07 00:31:05  11.905792  57.681060  265902200    Moored   \n             2017-05-07 01:01:05  11.905650  57.681127  265902200    Moored   \n             2017-05-07 01:07:05  11.905700  57.681107  265902200    Moored   \n\n                                  SOG    COG   ShipType   Distance  \ntraj_id      DateTime                                               \nAB RAMANTENN 2017-05-07 00:13:05  0.1  170.7  Undefined        NaN  \n             2017-05-07 00:25:04  0.1  170.7  Undefined   2.457384  \n             2017-05-07 00:31:05  0.1  177.4  Undefined   5.883613  \n             2017-05-07 01:01:05  0.0  175.6  Undefined  17.391237  \n             2017-05-07 01:07:05  0.1  180.8  Undefined   5.970428  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>lat</th>\n      <th>lon</th>\n      <th>MMSI</th>\n      <th>NavStatus</th>\n      <th>SOG</th>\n      <th>COG</th>\n      <th>ShipType</th>\n      <th>Distance</th>\n    </tr>\n    <tr>\n      <th>traj_id</th>\n      <th>DateTime</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">AB RAMANTENN</th>\n      <th>2017-05-07 00:13:05</th>\n      <td>11.905735</td>\n      <td>57.681092</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>170.7</td>\n      <td>Undefined</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:25:04</th>\n      <td>11.905740</td>\n      <td>57.681070</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>170.7</td>\n      <td>Undefined</td>\n      <td>2.457384</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:31:05</th>\n      <td>11.905792</td>\n      <td>57.681060</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>177.4</td>\n      <td>Undefined</td>\n      <td>5.883613</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 01:01:05</th>\n      <td>11.905650</td>\n      <td>57.681127</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.0</td>\n      <td>175.6</td>\n      <td>Undefined</td>\n      <td>17.391237</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 01:07:05</th>\n      <td>11.905700</td>\n      <td>57.681107</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>180.8</td>\n      <td>Undefined</td>\n      <td>5.970428</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Now using PTRAIL, generate distance features and\n",
    "# run hampel filter on the dataset to remove outliers.\n",
    "from ptrail.features.kinematic_features import KinematicFeatures\n",
    "from ptrail.preprocessing.filters import Filters\n",
    "\n",
    "dist_ships = KinematicFeatures.create_distance_column(np_ships)\n",
    "dist_ships.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of original DF: 84702\n",
      "Length of Filtered DF: 61394\n",
      "CPU times: user 171 ms, sys: 75.9 ms, total: 247 ms\n",
      "Wall time: 7.18 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yjharanwala/Desktop/PTRAIL/ptrail/preprocessing/filters.py:762: UserWarning: If kinematic features have been generated on the dataframe, then make sure to generate them again as outlier detection drops the point from the dataframe and does not run the kinematic features again.\n",
      "  warnings.warn(\"If kinematic features have been generated on the dataframe, then make \"\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Now, filter out the outliers using the hampel filter.\n",
    "\n",
    "filt_ships = Filters.hampel_outlier_detection(dist_ships,\n",
    "                                              column_name='Distance')\n",
    "print(f\"Length of original DF: {len(dist_ships)}\")\n",
    "print(f\"Length of Filtered DF: {len(filt_ships)}\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of original DF: 61394\n",
      "Length of Filtered DF: 61102\n"
     ]
    }
   ],
   "source": [
    "# Further more, remove the duplicate points from the\n",
    "# trajectories.\n",
    "\n",
    "fp_filt_ships = Filters.remove_duplicates(filt_ships)\n",
    "print(f\"Length of original DF: {len(filt_ships)}\")\n",
    "print(f\"Length of Filtered DF: {len(fp_filt_ships)}\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before: 61102\n",
      "After: 61097\n"
     ]
    }
   ],
   "source": [
    "#Now, remove the trajectories that have fewer than 3 points.\n",
    "print(f\"Before: {len(fp_filt_ships)}\")\n",
    "fp_filt_ships = Filters.remove_trajectories_with_less_points(fp_filt_ships)\n",
    "print(f\"After: {len(fp_filt_ships)}\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "                                  index        lat        lon       MMSI  \\\ntraj_id      DateTime                                                      \nAB RAMANTENN 2017-05-07 00:13:05      0  11.905735  57.681092  265902200   \n             2017-05-07 00:25:04      1  11.905740  57.681070  265902200   \n             2017-05-07 00:31:05      2  11.905792  57.681060  265902200   \n             2017-05-07 01:07:05      3  11.905700  57.681107  265902200   \n             2017-05-07 01:31:04      4  11.905708  57.681045  265902200   \n\n                                 NavStatus  SOG    COG  ShipType  Distance  \ntraj_id      DateTime                                                       \nAB RAMANTENN 2017-05-07 00:13:05    Moored  0.1  170.7         8       NaN  \n             2017-05-07 00:25:04    Moored  0.1  170.7         8  2.457384  \n             2017-05-07 00:31:05    Moored  0.1  177.4         8  5.883613  \n             2017-05-07 01:07:05    Moored  0.1  180.8         8  5.970428  \n             2017-05-07 01:31:04    Moored  0.1  173.2         8  6.804183  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>index</th>\n      <th>lat</th>\n      <th>lon</th>\n      <th>MMSI</th>\n      <th>NavStatus</th>\n      <th>SOG</th>\n      <th>COG</th>\n      <th>ShipType</th>\n      <th>Distance</th>\n    </tr>\n    <tr>\n      <th>traj_id</th>\n      <th>DateTime</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">AB RAMANTENN</th>\n      <th>2017-05-07 00:13:05</th>\n      <td>0</td>\n      <td>11.905735</td>\n      <td>57.681092</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>170.7</td>\n      <td>8</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:25:04</th>\n      <td>1</td>\n      <td>11.905740</td>\n      <td>57.681070</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>170.7</td>\n      <td>8</td>\n      <td>2.457384</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:31:05</th>\n      <td>2</td>\n      <td>11.905792</td>\n      <td>57.681060</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>177.4</td>\n      <td>8</td>\n      <td>5.883613</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 01:07:05</th>\n      <td>3</td>\n      <td>11.905700</td>\n      <td>57.681107</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>180.8</td>\n      <td>8</td>\n      <td>5.970428</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 01:31:04</th>\n      <td>4</td>\n      <td>11.905708</td>\n      <td>57.681045</td>\n      <td>265902200</td>\n      <td>Moored</td>\n      <td>0.1</td>\n      <td>173.2</td>\n      <td>8</td>\n      <td>6.804183</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now, since the model fitting does not take string values,\n",
    "# convert the ship types to integers.\n",
    "\n",
    "fp_filt_ships[\"ShipType\"] = fp_filt_ships[\"ShipType\"].str.strip()\n",
    "int_test = []\n",
    "types = fp_filt_ships['ShipType'].tolist()\n",
    "for i in range(len(fp_filt_ships['ShipType'])):\n",
    "    if types[i] == 'Tanker':\n",
    "        int_test.append(0)\n",
    "    elif types[i] == 'Passenger':\n",
    "        int_test.append(1)\n",
    "    elif types[i] == 'HSC':\n",
    "        int_test.append(2)\n",
    "    elif types[i] == 'Pilot':\n",
    "        int_test.append(3)\n",
    "    elif types[i] == 'SAR':\n",
    "        int_test.append(4)\n",
    "    elif types[i] == 'Tug':\n",
    "        int_test.append(5)\n",
    "    elif types[i] == 'Cargo':\n",
    "        int_test.append(6)\n",
    "    elif types[i] == 'Pleasure':\n",
    "        int_test.append(7)\n",
    "    elif types[i] == 'Undefined':\n",
    "        int_test.append(8)\n",
    "    elif types[i] == 'Sailing':\n",
    "        int_test.append(9)\n",
    "    elif types[i] == 'Law enforcement':\n",
    "        int_test.append(10)\n",
    "    elif types[i] == 'Spare 2':\n",
    "        int_test.append(11)\n",
    "    elif types[i] == 'Diving':\n",
    "        int_test.append(12)\n",
    "    elif types[i] == 'Fishing':\n",
    "        int_test.append(13)\n",
    "    elif types[i] == 'Other':\n",
    "        int_test.append(14)\n",
    "    else:\n",
    "        int_test.append(15)\n",
    "fp_filt_ships['ShipType'] = int_test\n",
    "fp_filt_ships.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['traj_id', 'DateTime', 'index', 'lat', 'lon', 'MMSI', 'NavStatus',\n",
      "       'SOG', 'COG', 'ShipType', 'Distance'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/plain": "                                        lat        lon  ShipType\ntraj_id      DateTime                                           \nAB RAMANTENN 2017-05-07 00:13:05  11.905735  57.681092         8\n             2017-05-07 00:25:04  11.905740  57.681070         8\n             2017-05-07 00:31:05  11.905792  57.681060         8\n             2017-05-07 01:07:05  11.905700  57.681107         8\n             2017-05-07 01:31:04  11.905708  57.681045         8",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>lat</th>\n      <th>lon</th>\n      <th>ShipType</th>\n    </tr>\n    <tr>\n      <th>traj_id</th>\n      <th>DateTime</th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">AB RAMANTENN</th>\n      <th>2017-05-07 00:13:05</th>\n      <td>11.905735</td>\n      <td>57.681092</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:25:04</th>\n      <td>11.905740</td>\n      <td>57.681070</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:31:05</th>\n      <td>11.905792</td>\n      <td>57.681060</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 01:07:05</th>\n      <td>11.905700</td>\n      <td>57.681107</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 01:31:04</th>\n      <td>11.905708</td>\n      <td>57.681045</td>\n      <td>8</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Finally, for cubic interpolation, 2 values having\n",
    "# same traj_id and datetime are not allowed. Hence\n",
    "# drop those points and convert the DF back to PTRAILDataFrame.\n",
    "\n",
    "fp_filt_ships = fp_filt_ships.reset_index().drop_duplicates(subset=['traj_id', 'DateTime'])\n",
    "print(fp_filt_ships.columns)\n",
    "fp_filt_ships = fp_filt_ships[['traj_id', 'DateTime', 'lat', 'lon', 'ShipType']]\n",
    "fp_filt_ships = PTRAILDataFrame(data_set=fp_filt_ships,\n",
    "                              latitude='lat',\n",
    "                              longitude='lon',\n",
    "                              datetime='DateTime',\n",
    "                              traj_id='traj_id')\n",
    "fp_filt_ships.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "False    47968\nTrue     12980\nName: DateTime, dtype: int64"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = fp_filt_ships.reset_index()[\"DateTime\"].diff().dt.total_seconds()\n",
    "(a > 30).value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of original DF: 60948\n",
      "Length of interpolated DF: 83662\n"
     ]
    }
   ],
   "source": [
    "# Now, interpolate the filtered dataframe and add points\n",
    "# to make the trajectories smoother.\n",
    "\n",
    "from ptrail.preprocessing.interpolation import Interpolation as ip\n",
    "\n",
    "ip_ships = ip.interpolate_position(fp_filt_ships,\n",
    "                                   time_jump=15,\n",
    "                                   ip_type='cubic')\n",
    "\n",
    "print(f\"Length of original DF: {len(fp_filt_ships)}\")\n",
    "print(f\"Length of interpolated DF: {len(ip_ships)}\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "                                        lat        lon  ShipType\ntraj_id      DateTime                                           \nAB RAMANTENN 2017-05-07 00:13:05  11.905735  57.681092         8\n             2017-05-07 00:13:20  11.905732  57.681092         8\n             2017-05-07 00:25:04  11.905740  57.681070         8\n             2017-05-07 00:25:19  11.905742  57.681069         8\n             2017-05-07 00:31:05  11.905792  57.681060         8",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>lat</th>\n      <th>lon</th>\n      <th>ShipType</th>\n    </tr>\n    <tr>\n      <th>traj_id</th>\n      <th>DateTime</th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">AB RAMANTENN</th>\n      <th>2017-05-07 00:13:05</th>\n      <td>11.905735</td>\n      <td>57.681092</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:13:20</th>\n      <td>11.905732</td>\n      <td>57.681092</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:25:04</th>\n      <td>11.905740</td>\n      <td>57.681070</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:25:19</th>\n      <td>11.905742</td>\n      <td>57.681069</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>2017-05-07 00:31:05</th>\n      <td>11.905792</td>\n      <td>57.681060</td>\n      <td>8</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now, fixing the ShipType column for the interpolated ships dataset.\n",
    "\n",
    "# The logic behind this is that when a point is added for a\n",
    "# particular traj_id, then it will retain the ship type as well\n",
    "# since interpolation is performed on each trajectory. As a result\n",
    "# the ShipType remains the same and they can be reassigned to\n",
    "# new points as well.\n",
    "\n",
    "# Create a list of all unique ids.\n",
    "ids_ = list(ip_ships.reset_index()['traj_id'].value_counts().keys())\n",
    "\n",
    "df_chunks = []\n",
    "# Create a small chunk for each ID, then for the same ID in the\n",
    "# original dataset, grab the species and then again assign that\n",
    "# to the interpolated ID.\n",
    "for i in range(len(ids_)):\n",
    "    small = ip_ships.reset_index().loc[ip_ships.reset_index()['traj_id'] == ids_[i]]\n",
    "    spec = fp_filt_ships.reset_index().loc[fp_filt_ships.reset_index()['traj_id'] == ids_[i], 'ShipType']\n",
    "    small['ShipType'] = spec.value_counts().idxmax()\n",
    "    df_chunks.append(small)\n",
    "\n",
    "# Now, convert the dataframe with interpolated species to\n",
    "# PTRAILDataFrame.\n",
    "ip_ships = PTRAILDataFrame(data_set=pd.concat(df_chunks),\n",
    "                         latitude='lat',\n",
    "                         longitude='lon',\n",
    "                         traj_id='traj_id',\n",
    "                         datetime='DateTime')\n",
    "ip_ships.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 510 ms, sys: 8.05 ms, total: 518 ms\n",
      "Wall time: 516 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import datetime as dt\n",
    "\n",
    "\n",
    "# Convert the timestamp to seconds.\n",
    "def dtt2timestamp(dtt):\n",
    "    ts = (dtt.hour * 60 + dtt.minute) * 60 + dtt.second\n",
    "    #if you want microseconds as well\n",
    "    ts += dtt.microsecond * 10**(-6)\n",
    "    return ts\n",
    "\n",
    "\n",
    "# Now, on both the train and test datasets, we generate\n",
    "# date and time features and then convert both values to ordinal\n",
    "# form in order to be eligible for model fitting.\n",
    "from ptrail.features.temporal_features import TemporalFeatures\n",
    "\n",
    "# 1. Train dataset.\n",
    "fp_filt_ships = TemporalFeatures.create_date_column(fp_filt_ships)\n",
    "fp_filt_ships = TemporalFeatures.create_time_column(fp_filt_ships)\n",
    "\n",
    "fp_filt_ships['Time'] = fp_filt_ships['Time'].apply(dtt2timestamp)\n",
    "fp_filt_ships['Date'] = fp_filt_ships['Date'].map(dt.datetime.toordinal)\n",
    "\n",
    "# 2. Test dataset.\n",
    "ip_ships = TemporalFeatures.create_time_column(ip_ships)\n",
    "ip_ships = TemporalFeatures.create_date_column(ip_ships)\n",
    "\n",
    "ip_ships['Date'] = ip_ships['Date'].map(dt.datetime.toordinal)\n",
    "ip_ships['Time'] = ip_ships['Time'].apply(dtt2timestamp)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [],
   "source": [
    "# Now, splitting the test datasets into 4 following parts:\n",
    "#   1. Training:\n",
    "#       1.1 train_x\n",
    "#       1.2 train_y\n",
    "#   2. Testing:\n",
    "#       2.1 test_x\n",
    "#       2.2 test_y\n",
    "\n",
    "train_x = fp_filt_ships.drop(columns=['ShipType'])\n",
    "train_y = fp_filt_ships.reset_index()['ShipType']\n",
    "\n",
    "test_x = ip_ships.drop(columns=['ShipType'])\n",
    "test_y = ip_ships.reset_index()['ShipType']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------- RandomForest Classifier -----------------\n",
      "The predicted train set values for RF are: [8 8 8 ... 9 9 9]\n",
      "The predicted test set values for RF are: [8 8 8 ... 1 1 0]\n",
      "\n",
      "The Training accuracy of RF is: 100.0%\n",
      "The Testing accuracy of RF is: 79.61%\n",
      "----------------------------------------------------------\n",
      "\n",
      "CPU times: user 7.66 s, sys: 48 ms, total: 7.71 s\n",
      "Wall time: 7.71 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Now it is time to train some models and predict\n",
    "# the ship types.\n",
    "\n",
    "# 1. RandomForestClassifier model.\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "rf_model = RandomForestClassifier()\n",
    "rf_model.fit(train_x, train_y)\n",
    "\n",
    "# Now, lets predict on the training and testing set\n",
    "# using the above trained model.\n",
    "rf_train_predict = rf_model.predict(train_x)\n",
    "rf_test_predict = rf_model.predict(test_x)\n",
    "\n",
    "\n",
    "# Finally, lets test the accuracy of the model on both\n",
    "# the datasets.\n",
    "rf_train_accuracy = accuracy_score(train_y, rf_train_predict)\n",
    "rf_test_accuracy = accuracy_score(test_y, rf_test_predict)\n",
    "\n",
    "print('---------------- RandomForest Classifier -----------------')\n",
    "print(f\"The predicted train set values for RF are: {rf_train_predict}\")\n",
    "print(f\"The predicted test set values for RF are: {rf_test_predict}\\n\")\n",
    "print(f\"The Training accuracy of RF is: {round(rf_train_accuracy*100, 2)}%\")\n",
    "print(f\"The Testing accuracy of RF is: {round(rf_test_accuracy*100, 2)}%\")\n",
    "print('----------------------------------------------------------\\n')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------- DecisionTree Classifier -----------------\n",
      "The predicted train set values for DT are: [8 8 8 ... 9 9 9]\n",
      "The predicted test set values for DT are: [8 8 8 ... 1 1 1]\n",
      "\n",
      "The Training accuracy of DT is: 100.0%\n",
      "The Testing accuracy of DT is: 78.51%\n",
      "----------------------------------------------------------\n",
      "\n",
      "CPU times: user 212 ms, sys: 9 µs, total: 212 ms\n",
      "Wall time: 210 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# 2. DecisionTree Classifier model.\n",
    "\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "dt_model = DecisionTreeClassifier()\n",
    "dt_model.fit(train_x, train_y)\n",
    "\n",
    "# Now, lets predict on the training and testing set\n",
    "# using the above trained model.\n",
    "dt_train_predict = dt_model.predict(train_x)\n",
    "dt_test_predict = dt_model.predict(test_x)\n",
    "\n",
    "\n",
    "# Finally, lets test the accuracy of the model on both\n",
    "# the datasets.\n",
    "dt_train_accuracy = accuracy_score(train_y, dt_train_predict)\n",
    "dt_test_accuracy = accuracy_score(test_y, dt_test_predict)\n",
    "\n",
    "print('---------------- DecisionTree Classifier -----------------')\n",
    "print(f\"The predicted train set values for DT are: {dt_train_predict}\")\n",
    "print(f\"The predicted test set values for DT are: {dt_test_predict}\\n\")\n",
    "print(f\"The Training accuracy of DT is: {round(dt_train_accuracy*100, 2)}%\")\n",
    "print(f\"The Testing accuracy of DT is: {round(dt_test_accuracy*100, 2)}%\")\n",
    "print('----------------------------------------------------------\\n')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------- Naive Bayes Classifier -----------------\n",
      "The predicted train set values for GNB are: [1 1 1 ... 0 0 0]\n",
      "The predicted test set values for GNB are: [6 6 6 ... 6 6 6]\n",
      "\n",
      "The Training accuracy of GNB is: 27.61%\n",
      "The Testing accuracy of GNB is: 4.25%\n",
      "----------------------------------------------------------\n",
      "\n",
      "CPU times: user 108 ms, sys: 8.01 ms, total: 116 ms\n",
      "Wall time: 115 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# 3. Gaussian Naive Bayes model.\n",
    "\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "gnb_model = GaussianNB()\n",
    "gnb_model.fit(train_x, train_y)\n",
    "\n",
    "# Now, lets predict on the training and testing set\n",
    "# using the above trained model.\n",
    "gnb_train_predict = gnb_model.predict(X=train_x)\n",
    "gnb_test_predict = gnb_model.predict(test_x)\n",
    "\n",
    "\n",
    "# Finally, lets test the accuracy of the model on both\n",
    "# the datasets.\n",
    "gnb_train_accuracy = accuracy_score(train_y, gnb_train_predict)\n",
    "gnb_test_accuracy = accuracy_score(test_y, gnb_test_predict)\n",
    "\n",
    "print('----------------- Naive Bayes Classifier -----------------')\n",
    "print(f\"The predicted train set values for GNB are: {gnb_train_predict}\")\n",
    "print(f\"The predicted test set values for GNB are: {gnb_test_predict}\\n\")\n",
    "print(f\"The Training accuracy of GNB is: {round(gnb_train_accuracy*100, 2)}%\")\n",
    "print(f\"The Testing accuracy of GNB is: {round(gnb_test_accuracy*100, 2)}%\")\n",
    "print('----------------------------------------------------------\\n')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------- KNN Classifier ---------------------\n",
      "The predicted train set values for KNN are: [8 8 0 ... 0 1 0]\n",
      "The predicted test set values for KNN are: [0 0 0 ... 0 0 0]\n",
      "\n",
      "The Training accuracy of KNN is: 45.32%\n",
      "The Testing accuracy of KNN is: 26.64%\n",
      "----------------------------------------------------------\n",
      "\n",
      "CPU times: user 5.83 s, sys: 4.1 ms, total: 5.83 s\n",
      "Wall time: 5.83 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# 4. K-Nearest Neighbors Classifier model.\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "knn_model = KNeighborsClassifier()\n",
    "knn_model.fit(train_x, train_y)\n",
    "\n",
    "# Now, lets predict on the training and testing set\n",
    "# using the above trained model.\n",
    "knn_train_predict = knn_model.predict(train_x)\n",
    "knn_test_predict = knn_model.predict(test_x)\n",
    "\n",
    "\n",
    "# Finally, lets test the accuracy of the model on both\n",
    "# the datasets.\n",
    "knn_train_accuracy = accuracy_score(train_y, knn_train_predict)\n",
    "knn_test_accuracy = accuracy_score(test_y, knn_test_predict)\n",
    "\n",
    "print('--------------------- KNN Classifier ---------------------')\n",
    "print(f\"The predicted train set values for KNN are: {knn_train_predict}\")\n",
    "print(f\"The predicted test set values for KNN are: {knn_test_predict}\\n\")\n",
    "print(f\"The Training accuracy of KNN is: {round(knn_train_accuracy*100, 2)}%\")\n",
    "print(f\"The Testing accuracy of KNN is: {round(knn_test_accuracy*100, 2)}%\")\n",
    "print('----------------------------------------------------------\\n')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------- K-Means Classifier --------------------\n",
      "The predicted train set values for KM are: [0 0 0 ... 6 6 6]\n",
      "The predicted test set values for KM are: [3 3 3 ... 3 3 3]\n",
      "\n",
      "The Training accuracy of KM is: 6.08%\n",
      "The Testing accuracy of KM is: 13.49%\n",
      "----------------------------------------------------------\n",
      "\n",
      "CPU times: user 19.9 s, sys: 595 ms, total: 20.5 s\n",
      "Wall time: 2.21 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# 5. K-Means Classifier model.\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "km_model = KMeans(15)\n",
    "km_model.fit(train_x, train_y)\n",
    "\n",
    "# Now, lets predict on the training and testing set\n",
    "# using the above trained model.\n",
    "km_train_predict = km_model.predict(train_x)\n",
    "km_test_predict = km_model.predict(test_x)\n",
    "\n",
    "\n",
    "# Finally, lets test the accuracy of the model on both\n",
    "# the datasets.\n",
    "km_train_accuracy = accuracy_score(train_y, km_train_predict)\n",
    "km_test_accuracy = accuracy_score(test_y, km_test_predict)\n",
    "\n",
    "print('------------------- K-Means Classifier --------------------')\n",
    "print(f\"The predicted train set values for KM are: {km_train_predict}\")\n",
    "print(f\"The predicted test set values for KM are: {km_test_predict}\\n\")\n",
    "print(f\"The Training accuracy of KM is: {round(km_train_accuracy*100, 2)}%\")\n",
    "print(f\"The Testing accuracy of KM is: {round(km_test_accuracy*100, 2)}%\")\n",
    "print('----------------------------------------------------------\\n')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}